﻿using System;
using System.Collections.Generic;

namespace MachineLearning.BackPropagatingNeuralNetwork
{
    public class NeuralNetwork
    {
        //Number of neurons in each layer
        private int numInput;
        private int numHidden;
        private int numOutput;

        //Inner weights, biases and outputs
        private double[] inputs;
        private double[][] ihWeights; 
        private double[] ihSums;
        private double[] ihBiases;
        private double[] ihOutputs;
        private double[][] hoWeights; 
        private double[] hoSums;
        private double[] hoBiases;
        private double[] outputs;
        private double[] outputGrads; 
        private double[] hiddenGrads; 
        private double[][] ihPrevWeightsDelta; 
        private double[] ihPrevBiasesDelta;
        private double[][] hoPrevWeightsDelta;
        private double[] hoPrevBiasesDelta;

        //Hyperparameters
        public double LearningRate { get; set; }
        public double Momentum { get; set; }
        public bool IsComplete { get; set; }
        public int MaxRun { get; set; }
        public double Tolerance { get; set; }

        public NeuralNetwork(int numInput, int numHidden, int numOutput)
        {
            this.numInput = numInput;
            this.numHidden = numHidden;
            this.numOutput = numOutput;

            inputs = new double[numInput];
            ihWeights = NeuralNetworkUtility.MakeMatrix(numInput, numHidden);
            ihSums = new double[numHidden];
            ihBiases = new double[numHidden];
            ihOutputs = new double[numHidden];
            hoWeights = NeuralNetworkUtility.MakeMatrix(numHidden, numOutput);
            hoSums = new double[numOutput];
            hoBiases = new double[numOutput];
            outputs = new double[numOutput];
            outputGrads = new double[numOutput];
            hiddenGrads = new double[numHidden];
            ihPrevWeightsDelta = NeuralNetworkUtility.MakeMatrix(numInput, numHidden);
            ihPrevBiasesDelta = new double[numHidden];
            hoPrevWeightsDelta = NeuralNetworkUtility.MakeMatrix(numHidden, numOutput);
            hoPrevBiasesDelta = new double[numOutput];

            LearningRate = 0.6;
            Momentum = 0.04;
            MaxRun = 1000;
            Tolerance = 0.001;
        }

        public NeuralNetwork(NeuralNetworkModel model)
            : this(model.NumInputs, model.NumHidden, model.NumOutputs)
        {
            this.LearningRate = model.LearningRate;
            this.Momentum = model.Momentum;
            this.MaxRun = model.MaxRun;
            this.Tolerance = model.Tolerance;

            setWeights(model.Weights);
        }

        public NeuralNetworkModel Model
        {
            get
            {
                return new NeuralNetworkModel()
                {
                    NumInputs = numInput,
                    NumHidden = numHidden,
                    NumOutputs = numOutput,
                    Tolerance = this.Tolerance,
                    MaxRun = this.MaxRun,
                    LearningRate = this.LearningRate,
                    Momentum = this.Momentum,
                    Weights = getWeights()
                };
            }
        }

        public void Train(List<double[]> inputs, List<double[]> targets)
        {
            var run = 0;
            var err = double.PositiveInfinity;

            initWeights();

            while (err > Tolerance && run <= MaxRun)
            {
                for (int i = 0; i < inputs.Count; ++i)
                {
                    var result = ComputeOutputs(inputs[i]);

                    err = NeuralNetworkMath.TotalError(targets[i], result);
                    updateWeights(targets[i]);
                }

                run++;
            }

            IsComplete = true;
        }
        
        //Used for loading weights
        private void setWeights(double[] weights)
        {
            int numWeights = (numInput * numHidden) + (numHidden * numOutput) + numHidden + numOutput;

            if (weights.Length != numWeights)
                throw new Exception("The weights array length: " + weights.Length + " does not match the total number of weights and biases: " + numWeights);

            int k = 0;

            for (int i = 0; i < numInput; ++i)
                for (int j = 0; j < numHidden; ++j)
                    ihWeights[i][j] = weights[k++];

            for (int i = 0; i < numHidden; ++i)
                ihBiases[i] = weights[k++];

            for (int i = 0; i < numHidden; ++i)
                for (int j = 0; j < numOutput; ++j)
                    hoWeights[i][j] = weights[k++];

            for (int i = 0; i < numOutput; ++i)
                hoBiases[i] = weights[k++];
        }

        //Used for saving weights
        private double[] getWeights()
        {
            int numWeights = (numInput * numHidden) + (numHidden * numOutput) + numHidden + numOutput;
            double[] result = new double[numWeights];
            int k = 0;

            for (int i = 0; i < ihWeights.Length; ++i)
                for (int j = 0; j < ihWeights[0].Length; ++j)
                    result[k++] = ihWeights[i][j];

            for (int i = 0; i < ihBiases.Length; ++i)
                result[k++] = ihBiases[i];

            for (int i = 0; i < hoWeights.Length; ++i)
                for (int j = 0; j < hoWeights[0].Length; ++j)
                    result[k++] = hoWeights[i][j];

            for (int i = 0; i < hoBiases.Length; ++i)
                result[k++] = hoBiases[i];

            return result;
        }

        public double[] ComputeOutputs(double[] xValues)
        {
            if (xValues.Length != numInput)
                throw new Exception("Inputs array length " + inputs.Length + " does not match NN numInput value " + numInput);

            for (int i = 0; i < numHidden; ++i)
                ihSums[i] = 0.0;

            for (int i = 0; i < numOutput; ++i)
                hoSums[i] = 0.0;

            for (int i = 0; i < xValues.Length; ++i)
                this.inputs[i] = xValues[i];

            for (int j = 0; j < numHidden; ++j)
                for (int i = 0; i < numInput; ++i)
                    ihSums[j] += this.inputs[i] * ihWeights[i][j];

            for (int i = 0; i < numHidden; ++i)
                ihSums[i] += ihBiases[i];

            for (int i = 0; i < numHidden; ++i)
                ihOutputs[i] = NeuralNetworkMath.SigmoidFunction(ihSums[i]);

            for (int j = 0; j < numOutput; ++j)
                for (int i = 0; i < numHidden; ++i)
                    hoSums[j] += ihOutputs[i] * hoWeights[i][j];

            for (int i = 0; i < numOutput; ++i)
                hoSums[i] += hoBiases[i];

            for (int i = 0; i < numOutput; ++i)
                this.outputs[i] = NeuralNetworkMath.HyperTanFunction(hoSums[i]);

            var result = new double[numOutput];
            this.outputs.CopyTo(result, 0);

            return result;
        }

        private void updateWeights(double[] tValues)
        {
            if (tValues.Length != numOutput)
                throw new Exception("target values not same Length as output in UpdateWeights");

            for (int i = 0; i < outputGrads.Length; ++i)
            {
                var derivative = NeuralNetworkMath.DerivateOfTanh(outputs[i]);
                outputGrads[i] = derivative * (tValues[i] - outputs[i]);
            }

            for (int i = 0; i < hiddenGrads.Length; ++i)
            {
                var derivative = NeuralNetworkMath.DerivativeOfSigmoid(ihOutputs[i]);
                var sum = 0.0;
                for (int j = 0; j < numOutput; ++j)
                    sum += outputGrads[j] * hoWeights[i][j]; 
                hiddenGrads[i] = derivative * sum;
            }

            for (int i = 0; i < ihWeights.Length; ++i) 
            {
                for (int j = 0; j < ihWeights[0].Length; ++j) 
                {
                    var delta = getDelta(hiddenGrads[j], inputs[i]);
                    ihWeights[i][j] += delta;
                    ihWeights[i][j] += Momentum * ihPrevWeightsDelta[i][j];
                }
            }

            for (int i = 0; i < ihBiases.Length; ++i)
            {
                var delta = getDelta(hiddenGrads[i], 1);
                ihBiases[i] += delta;
                ihBiases[i] += Momentum * ihPrevBiasesDelta[i];
            }

            for (int i = 0; i < hoWeights.Length; ++i) 
            {
                for (int j = 0; j < hoWeights[0].Length; ++j)
                {
                    var delta = getDelta(outputGrads[j], ihOutputs[i]);
                    hoWeights[i][j] += delta;
                    hoWeights[i][j] += Momentum * hoPrevWeightsDelta[i][j];
                    hoPrevWeightsDelta[i][j] = delta;
                }
            }

            for (int i = 0; i < hoBiases.Length; ++i)
            {
                var delta = getDelta(outputGrads[i], 1);
                hoBiases[i] += delta;
                hoBiases[i] += Momentum * hoPrevBiasesDelta[i];
                hoPrevBiasesDelta[i] = delta;
            }
        }
        
        private void initWeights()
        {
            var weights = new List<double>();
            var random = new Random();

            for (int i = 0; i < numInput * numHidden; ++i)
                weights.Add(random.NextDouble() * 2);
            for (int i = 0; i < numHidden; ++i)
                weights.Add(random.NextDouble() * -10);
            for (int i = 0; i < numHidden * numOutput; ++i)
                weights.Add(random.NextDouble() * 2);
            for (int i = 0; i < numOutput; ++i)
                weights.Add(random.NextDouble() * -10);

            setWeights(weights.ToArray());
        }

        private double getDelta(double gradient, double input)
        {
            return LearningRate * gradient * input;
        }
    }

    static class NeuralNetworkMath
    {
        public static double SigmoidFunction(double x)
        {
            return 1 / (1 + Math.Exp(-x));
        }

        public static double HyperTanFunction(double x)
        {
            return Math.Tanh(x);
        }

        public static double DerivativeOfSigmoid(double x)
        {
            return (1 - x) * x;
        }

        public static double DerivateOfTanh(double x)
        {
            return (1 - x) * (1 + x);
        }

        public static double TotalError(double[] targets, double[] outputs)
        {
            var sum = 0.0;
            for (int i = 0; i < targets.Length; ++i)
                sum += Math.Abs(targets[i] - outputs[i]);
            return sum;
        }
    }
}

